package com.feidee.fd.sml.algorithm.feature

import com.feidee.fd.sml.algorithm.component.feature.{TokenizeEncoder, TokenizeEncoderParam}
import com.feidee.fd.sml.algorithm.util.{TestingDataGenerator, ToolClass}
import org.scalatest.FunSuite

/**
  * @Author JunxinWang
  * @Date 2019/3/25 18:54
  * @Description 测试TokenizerEncoder
  * @Reviewer YongChen
  */
class TokenizeEncoderSuite extends FunSuite {

  val tokenizer = new TokenizeEncoder()
  val paramStr: String =
    """
      |{
      | "inputCol": "feature" ,
      | "outputCol": "words" ,
      | "pattern" : "\\w+"
      |}
    """.stripMargin


  val param: TokenizeEncoderParam = tokenizer.parseParam(new ToolClass().encrypt(paramStr))

  test("tokenizer parameter construction test") {
    assert(param.input_pt == null && param.output_pt == null && param.modelPath == null &&
      "feature".equals(param.inputCol)
    && !param.gaps
      && param.minTokenLength==1
      && "\\w+".equals(param.pattern)
      && param.toLowercase
    )
  }

  test("tokenizer transforming function test") {

    val sentenceDataFrame = TestingDataGenerator.spark.createDataFrame(Seq(
      (0, "Hi I heard about Spark"),
      (1, "I wish Java could use case classes"),
      (2, "Logistic,regression,models,are,neat")
    )).toDF("id", "feature")
    val tokenizerData = tokenizer.train(param,sentenceDataFrame).transform(sentenceDataFrame)

    tokenizerData.show(false)

  }

}

